package com.kingsoft.dc.khaos.module.spark.util;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.io.IOUtils;
import org.apache.spark.sql.SaveMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Created by jing on 18/12/11.
 */
public class HdfsUtils {
    private static Logger log = LoggerFactory.getLogger(HdfsUtils.class);

    public static Map<String, Long> listFilesInfo(String hdfsPath) {
        FileSystem fs = null;
        Map<String, Long> fileMap = new HashMap<>();
        try {
            fs = FileSystem.get(new Configuration());
            FileStatus[] files = fs.globStatus(new Path(hdfsPath));
            for (FileStatus status : files) {
                if (status.isDirectory()) {
                    RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(new Path(status.getPath().toString()), false);
                    while (iterator.hasNext()) {
                        LocatedFileStatus fileStatus = iterator.next();
                        Path fullPath = fileStatus.getPath();
                        long currLength = fileStatus.getLen();
                        fileMap.put(fullPath.toString(), currLength);
                    }
                } else {
                    long currLength = status.getLen();
                    fileMap.put(status.getPath().toString(), currLength);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return fileMap;
    }

    /**
     * 遍历文件
     *
     * @param hdfsPath
     * @param filterPreffix 用来过滤SUCCESS文件
     * @param fs
     * @return
     */
    public static List<String> listFiles(String hdfsPath, String filterPreffix, FileSystem fs) {
        List<String> fileList = new ArrayList<>();
        try {
            FileStatus[] files = fs.globStatus(new Path(hdfsPath));
            for (FileStatus status : files) {
                if (status.isDirectory()) {
                    RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(new Path(status.getPath().toString()), false);
                    while (iterator.hasNext()) {
                        LocatedFileStatus fileStatus = iterator.next();
                        Path fullPath = fileStatus.getPath();
                        String fileName = fullPath.getName();
                        if (fileName.startsWith(filterPreffix)) {
                            continue;
                        }
                        fileList.add(fileName);
                        log.info("file:{}", fileName);
                    }
                } else {
                    String fileName = status.getPath().getName();
                    if (!fileName.startsWith(filterPreffix)) {
                        fileList.add(fileName);
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return fileList;
    }

    /**
     * 重命名hdfs文件
     *
     * @param toRenameFile
     * @param renamedFile
     * @return
     */
    public static void renameFile(String toRenameFile, String renamedFile, FileSystem fs) {
        try {
            Path renamedPath = new Path(renamedFile);
            if (!fs.exists(renamedPath.getParent())) {
                fs.mkdirs(renamedPath.getParent());
            }
            fs.rename(new Path(toRenameFile), new Path(renamedFile));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 重命名hdfs文件
     *
     * @param sourcePathName 源目录
     * @param targetPathName 目标目录
     * @return
     */
    public static void moveFiles(String sourcePathName, String targetPathName, FileSystem fs) {
        try {
            List<String> files = listFiles(sourcePathName, "_", fs);
            Path targetPath = new Path(targetPathName);
            if (!fs.exists(targetPath)) {
                fs.mkdirs(targetPath);
            }
            for (String file : files) {
                fs.rename(new Path(sourcePathName + "/" + file), new Path(targetPathName + "/" + file));
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 删除集群上目录或文件
     *
     * @param inputPath
     */
    public static void deletePath(String inputPath, FileSystem fs) {
        try {
            fs.delete(new Path(inputPath), true);
        } catch (Exception e) {
            e.printStackTrace();
            log.warn("tmp dir remove failed.");
        }
    }
    //在指定位置新建一个文件，并写入字符
    public static void writeToHDFS(String hdfspath, String words, FileSystem fs, SaveMode saveMode,String charset) throws IOException, URISyntaxException
    {
        FSDataOutputStream out =null;
        try{
            Path path = new Path(hdfspath);
            if(saveMode.equals(SaveMode.Overwrite) && fs.exists(path))
                fs.delete(path,true);
                out = fs.create(path);   //创建文件
            out.write(words.getBytes(charset));
        }finally {
            out.close();
        }
    }


    /**
     * 追加文件内容
     *
     * @param sourcePath
     * @param targetPath
     * @param fs
     */
    public static void appendFile(String sourcePath, String targetPath, FileSystem fs) {
        FSDataInputStream in = null;
        FSDataOutputStream out = null;
        try {
            //要追加的文件流，inpath为文件
            in = fs.open(new Path(sourcePath));
            out = fs.append(new Path(targetPath));
            IOUtils.copyBytes(in, out, 4096, true);
        } catch (Exception e) {
            e.printStackTrace();
            if (in != null) {
                try {
                    in.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
            if (out != null) {
                try {
                    in.close();
                } catch (IOException e1) {
                    e1.printStackTrace();
                }
            }
        }
    }

    /**
     * 判断文件或者目录是否存在
     *
     * @param inputPath
     */
    public static Boolean isExists(String inputPath, FileSystem fs) {
        Boolean status = false;
        try {
            Path targetPath = new Path(inputPath);
            status = fs.exists(targetPath);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return status;
    }


    /**
     * 获取文件流对象
     * @param filePath
     */
    public static InputStream getFileInputStream(String filePath , FileSystem fs) throws IOException {
        FSDataInputStream in = null;
        in = fs.open(new Path(filePath));
        return in;
    }
}
